{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-08T15:14:53.864790Z",
     "start_time": "2025-10-08T15:14:43.327743Z"
    }
   },
   "source": [
    "import jieba\n",
    "import re\n",
    "from collections import Counter\n",
    "import pycorrector"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\JNU\\Project\\Python\\2025FMa\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c7fcf3827a91bffc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-08T07:54:14.985998Z",
     "start_time": "2025-10-08T07:54:14.904511Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   ItemID  Sentiment SentimentSource  \\\n",
      "0       1          0    Sentiment140   \n",
      "1       2          0    Sentiment140   \n",
      "2       3          1    Sentiment140   \n",
      "3       4          0    Sentiment140   \n",
      "4       5          0    Sentiment140   \n",
      "\n",
      "                                       SentimentText  \n",
      "0                       is so sad for my APL frie...  \n",
      "1                     I missed the New Moon trail...  \n",
      "2                            omg its already 7:30 :O  \n",
      "3            .. Omgaga. Im sooo  im gunna CRy. I'...  \n",
      "4           i think mi bf is cheating on me!!!   ...  \n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./data/twitter_data.csv')\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bc384a87-01f4-444b-9af6-ced0b642cb5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       SentimentText  word_count\n",
      "0                       is so sad for my APL frie...          28\n",
      "1                     I missed the New Moon trail...          25\n",
      "2                            omg its already 7:30 :O          19\n",
      "3            .. Omgaga. Im sooo  im gunna CRy. I'...          36\n",
      "4           i think mi bf is cheating on me!!!   ...          24\n"
     ]
    }
   ],
   "source": [
    "df['word_count']=df['SentimentText'].apply(lambda x:len(str(x).split(\" \"))) \n",
    "print(df[['SentimentText','word_count']].head()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7c7c7cd7-cfc1-47e2-bebd-4712c032e51c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       SentimentText  char_count\n",
      "0                       is so sad for my APL frie...          61\n",
      "1                     I missed the New Moon trail...          51\n",
      "2                            omg its already 7:30 :O          37\n",
      "3            .. Omgaga. Im sooo  im gunna CRy. I'...         132\n",
      "4           i think mi bf is cheating on me!!!   ...          53\n"
     ]
    }
   ],
   "source": [
    "df['char_count']= df['SentimentText'].str.len() \n",
    "print(df[['SentimentText','char_count']].head()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "363c84c3-bfdc-4557-b915-61d685f839b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       SentimentText  avg_word\n",
      "0                       is so sad for my APL frie...  4.857143\n",
      "1                     I missed the New Moon trail...  4.500000\n",
      "2                            omg its already 7:30 :O  3.800000\n",
      "3            .. Omgaga. Im sooo  im gunna CRy. I'...  3.880000\n",
      "4           i think mi bf is cheating on me!!!   ...  3.333333\n"
     ]
    }
   ],
   "source": [
    "def avg_word(sentence): \n",
    "    words=sentence.split()     \n",
    "    return (sum(len(word) for word in words)/len(words)) \n",
    "df['avg_word']=df['SentimentText'].apply(lambda x:avg_word(x)) \n",
    "print(df[['SentimentText','avg_word']].head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1a1b61fc-7d34-4209-a170-699e03797426",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0             is so sad for my apl friend.............\n",
      "1                     i missed the new moon trailer...\n",
      "2                              omg its already 7:30 :o\n",
      "3    .. omgaga. im sooo im gunna cry. i've been at ...\n",
      "4               i think mi bf is cheating on me!!! t_t\n",
      "Name: SentimentText, dtype: object\n"
     ]
    }
   ],
   "source": [
    "df['SentimentText']=df['SentimentText'].apply(lambda sen:\" \".join(x.lower() \n",
    "for x in sen.split()))  \n",
    "print(df['SentimentText'].head()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0c5e572a-eac8-44a5-9909-1c7ed198c895",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0                          is so sad for my apl friend\n",
      "1                        i missed the new moon trailer\n",
      "2                               omg its already 7 30 o\n",
      "3    omgaga im sooo im gunna cry i ve been at this ...\n",
      "4                  i think mi bf is cheating on me t_t\n",
      "Name: SentimentText, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def clean_text(text:str):\n",
    "    if pd.isna(text):\n",
    "        return text\n",
    "    cleaned = re.sub(r'\\W+', ' ', str(text))\n",
    "    return cleaned.strip()\n",
    "df['SentimentText'] = df['SentimentText'].apply(clean_text)\n",
    "print(df['SentimentText'].head()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "060920f4-6b57-40a6-9d5e-fa326146a5b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✓ 'stopwords' found at corpora/stopwords\n",
      "0                                       sad apl friend\n",
      "1                              missed new moon trailer\n",
      "2                                     omg already 7 30\n",
      "3    omgaga im sooo im gunna cry dentist since 11 s...\n",
      "4                             think mi bf cheating t_t\n",
      "Name: SentimentText, dtype: object\n"
     ]
    }
   ],
   "source": [
    "from nltk_manager import NLTKManager\n",
    "nltk_manager = NLTKManager('../.venv/nltk_data')\n",
    "stop = nltk_manager.get_stopwords('english')\n",
    "df['SentimentText'] = df['SentimentText'].apply(lambda sen: ' '.join(x for x in sen.split() if x not in stop))\n",
    "print(df['SentimentText'].head()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "06eef623-0913-43cc-a072-1d6ee3b82747",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uniform     1\n",
      "joke        1\n",
      "even        1\n",
      "hmmmm       1\n",
      "wonder      1\n",
      "number      1\n",
      "must        1\n",
      "positive    1\n",
      "thanks      1\n",
      "omg         1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "freq = pd.Series(' '.join(df['SentimentText']).split()).value_counts()[-10:] \n",
    "print(freq) \n",
    "df['SentimentText'] = df['SentimentText'].apply(lambda x: \" \".join(x for x in \n",
    "x.split() if x not in freq)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "929fe7c8-3f2d-425c-bb71-0ec65a2d233b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5                        worry much\n",
      "6    juuuuuuuuuuuuuuuuussssst chill\n",
      "7    sunny work tomorrow to tonight\n",
      "8         handed today miss already\n",
      "9                                  \n",
      "Name: SentimentText, dtype: object\n"
     ]
    }
   ],
   "source": [
    "from textblob import TextBlob  \n",
    "print(df['SentimentText'][5:10].apply(lambda x: str(TextBlob(x).correct()))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "001ea8e2-abb2-4f58-944a-42cce7be709a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
